package cn.cas.mango.service;

import cn.cas.mango.dto.HotSpotNews;
import cn.cas.mango.dto.Message;
import cn.cas.mango.dto.param.hot.HotMissionParam;
import cn.cas.mango.dto.param.hot.HotSpotParam;
import cn.cas.mango.service.samenews.SameHotNewsService;
import cn.cas.mango.util.Pair;
import cn.cas.mango.util.RenderDateUtil;
import cn.cas.mango.util.RequestUtil.BuildHotSpotRequest;
import cn.cas.mango.util.ResponseUtil.ResponseUtil;
import cn.cas.mango.util.TimeUtil;
import cn.cas.mango.util.dbscanHot.Data;
import cn.cas.mango.util.paramCheck.HotMissionParamCheck;
import cn.cas.mango.util.paramCheck.HotSpotParamCheckUtil;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.SortOptions;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch.core.CountRequest;
import co.elastic.clients.elasticsearch.core.CountResponse;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.core.search.TotalHits;
import co.elastic.clients.json.JsonData;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.utils.DateUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;

import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

// todo 在从数据库中获取数据的时候，没有考虑数量，应该分批获取数据。
@Service
@Slf4j
public class HotSpotSearchService {
  // 客户端操作
  @Autowired
  private ElasticsearchClient elasticsearchClient;
  // 基本配置项application.properties中的
  @Autowired
  private ConfigService config;
  @Autowired
  private BuildHotSpotRequest buildRequest;
  @Autowired
  private RequestService requestService;

  private int maxSize = 10000;

  private long getDateCount(HotSpotParam queryParam) throws IOException {
    CountRequest.Builder countRequest = buildRequest.getDateTotalNum(queryParam);
    CountResponse count = elasticsearchClient.count(countRequest.build());
    return count.count();
  }

  Pair<TotalHits, List<Hit<HotSpotNews>>> getBatchResponse(SearchRequest.Builder searchRequest1,
                                                           SearchRequest.Builder searchRequest2,
                                                           int size) throws IOException {
    SearchRequest build = searchRequest1.build();
    SearchResponse<HotSpotNews> search = elasticsearchClient.search(build, HotSpotNews.class);
    TotalHits total = search.hits().total();
    List<Hit<HotSpotNews>> hits = search.hits().hits();
    log.info("query str: {}", build);
    return new Pair<>(total, hits);
//    return requestService.getLarge(searchRequest1, searchRequest2, HotSpotNews.class, size);
  }

  private List<String> getFilteredKeywords(HotSpotNews news) {
    return news.getFilteredKeywords(config.getBlackList());
  }
  private JSONObject globalCluster(HotSpotParam queryParam) throws IOException {
    SortOptions sortOptions = SortOptions.of(
        s -> s.field(f -> f.field("fssj").order(SortOrder.Desc)));
    SearchRequest.Builder searchRequest1 = buildRequest.buildRequest(queryParam)
        .sort(sortOptions);
    SearchRequest.Builder searchRequest2 = buildRequest.buildRequest(queryParam)
        .sort(sortOptions);
    // 2.处理参数转换为对应的请求
    // 先求相似度，再聚类

    long searchStart = System.currentTimeMillis();
    TotalHits total; List<Hit<HotSpotNews>> response;
    Pair<TotalHits, List<Hit<HotSpotNews>>> batchResponse = getBatchResponse(searchRequest1, searchRequest2, -1);
    total = batchResponse.getKey(); response = batchResponse.getValue();
    List<Hit<HotSpotNews>> list = distinctById(response);
    long searchEnd = System.currentTimeMillis();
    log.info("hit number: {}, search time: {}s", total, (searchEnd - searchStart) / 1000.0);
    Map<String, Hit<HotSpotNews>> map2DepartmentNews = new HashMap<>();
    Map<String, Hit<HotSpotNews>> map2ProvinceNews = new HashMap<>();
    Set<String> provinces = config.getProvince2Nm().keySet();

    for (Hit<HotSpotNews> h: list) {
      assert h.source() != null;
      HotSpotNews source = h.source();
      String lyss = (String) source.getFromProvince();

      source.setNewsKeywords(getFilteredKeywords(source));
      if (provinces.contains(lyss)) {
        map2ProvinceNews.put(source.getId(), h);
      } else {
        map2DepartmentNews.put(source.getId(), h);
      }
    }
    TimeUtil timeUtil = new TimeUtil();

    Map<String, List<Hit<HotSpotNews>>> sameDepartmentNewsMap
//        = new SameHotNewsService(new ArrayList<>(map2DepartmentNews.values())).getSameNewsList(queryParam.getSimWeight(), queryParam.getEntityWeight());
        = wrapDepartment(map2DepartmentNews);
    log.info("show number: {}, cluster time: {}s", sameDepartmentNewsMap.size(), timeUtil.getCost());
    timeUtil = new TimeUtil();
    Map<String, List<Hit<HotSpotNews>>> sameProvinceNewsMap
        = new SameHotNewsService(new ArrayList<>(map2ProvinceNews.values())).getSameNewsList(queryParam.getSimWeight(), queryParam.getEntityWeight());

    log.info("show number: {}, cluster time: {}s", sameProvinceNewsMap.size(), timeUtil.getCost());
    // 放簇
    JSONObject data = new JSONObject();
    Pair<Map<String, Hit<HotSpotNews>>, Map<String, Hit<HotSpotNews>>> map2News = new Pair<>(map2ProvinceNews, map2DepartmentNews);
    Pair<Map<String, List<Hit<HotSpotNews>>>, Map<String, List<Hit<HotSpotNews>>>> sameNewsMap = new Pair<>(sameProvinceNewsMap, sameDepartmentNewsMap);
    data.put("data", processDoc(map2News, sameNewsMap));
    // normal
    {
      data.put("total", sameDepartmentNewsMap.size() + sameProvinceNewsMap.size());
      data.put("alive", true);
      data.put("sim", queryParam.getSimWeight());
      data.put("entity", queryParam.getEntityWeight());
      data.put("count", list.size());
      data.put("currentPage", queryParam.getCurrentPage());
      data.put("pageSize", queryParam.getPageSize());
    }
//    log.info(String.valueOf(data));
    //包装查询结果
    return data;
  }

  Map<String, List<Hit<HotSpotNews>>> wrapDepartment(Map<String, Hit<HotSpotNews>> map2DepartmentNews) {
    Map<String, List<Hit<HotSpotNews>>> ret = new LinkedHashMap<>();
    for (String string: map2DepartmentNews.keySet()) {
      ret.put(string, new ArrayList<>());
    }
    return ret;
  }


  private boolean judgeIncrementCluster(HotSpotParam queryParam, int nowTotal) {
    String startDate = queryParam.getStartDate();
    String endDate = queryParam.getEndDate();
    CountRequest countRequest = CountRequest.of(x -> x.index(config.getHotspotIndex())
        .query(q -> q.bool(f -> f.filter(r -> r.range(t -> t.field("fssj")
            .gte(JsonData.of(startDate))
            .lte(JsonData.of(endDate))
        ))
            .mustNot(k->k.terms(t->t.field("lyss").terms(m->m.value(buildRequest.getExclusiveProvince())
        ))))));
    try {
      CountResponse count = elasticsearchClient.count(countRequest);
      return count.count() > nowTotal;
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private JSONObject getResponse(HotSpotParam queryParam) {
    TimeUtil timeUtil = new TimeUtil();

    String requestNm = queryParam.getNm();
    String method = "xgs";
    String url = config.getTemporarySaveNewsUrl();
//    HttpHeaders headers = new HttpHeaders();
//    headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
    RestTemplate restTemplate = new RestTemplate();
////    MultiValueMap<String, String> paramMap = new LinkedMultiValueMap<>();
////    paramMap.add("method", method);
////    paramMap.add("rwnm", requestNm);
//    // Create the HttpEntity containing the form parameters
////    HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(paramMap, headers);
//    url = url + "?method=" + method + "&rwnm=" + requestNm;
//    ResponseEntity<JSONObject> forEntity = restTemplate.getForEntity(url, JSONObject.class);
    UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(url)
        .queryParam("method", method) // 替换为实际参数值
        .queryParam("rwnm", requestNm); // 替换为实际参数值

    // 设置请求头
    HttpHeaders headers = new HttpHeaders();
    headers.set("Content-Type", "application/x-www-form-urlencoded");

    // 创建HttpEntity对象，包含请求头
    HttpEntity<String> entity = new HttpEntity<>(headers);

    // 构建带有参数的URL字符串
    String urlWithParams = builder.toUriString();

    // 发起GET请求，带有请求头和参数
    ResponseEntity<String> response = restTemplate.exchange(
        urlWithParams,
        HttpMethod.GET,
        entity,
        String.class
    );
    log.info("get 28s cost: {}s", timeUtil.getCost());
    return JSONObject.parseObject(response.getBody());
  }

  private int getNowTotal(Set<String> distanceDates, JSONObject jsonObject) {

    JSONArray data = jsonObject.getJSONArray("data");
    AtomicInteger count = new AtomicInteger();
    for (Object o: data) {
      JSONObject d = (JSONObject) o;
      JSONArray event = d.getJSONArray("event");
      event.forEach(x->{
        JSONObject obj = (JSONObject)x;
        String date = (String) obj.get("fssj");
        if (distanceDates.contains(date)) {
          count.getAndIncrement();
        }
      });
    }
    return count.get();
  }

  /**
   * 如果需要对聚类后的结果重新聚类
   */
  private void rebuildIncrement() {

  }


  private Set<String> getInTempIds(Set<String> distanceDates,
                                   JSONObject dataJSONObject) {
    JSONArray dataArray = dataJSONObject.getJSONArray("data");
    Set<String> ret = new HashSet<>();
    for (Object o: dataArray) {
      JSONObject d = (JSONObject) o;
      JSONArray event = d.getJSONArray("event");
      for (Object e: event) {
        JSONObject jsonObject = (JSONObject)e;
        String date = jsonObject.getString("fssj");
        if (distanceDates.contains(date)) {
          ret.add(jsonObject.getString("nm"));
        }
      }
    }
    return ret;
  }

  private Map<String, JSONArray> getPastCast(JSONObject dataJSONObject) {
    JSONArray dataArray = dataJSONObject.getJSONArray("data");
    Map<String, JSONArray> ret = new HashMap<>();
    for (Object o: dataArray) {
      JSONObject jsonObject = (JSONObject) o;
      JSONArray events = jsonObject.getJSONArray("event");
      String id = ((JSONObject) events.get(0)).getString("nm");
      ret.put(id, events);
    }
    return ret;
  }

  private List<HotSpotNews> getNewsFromIdsArray(List<String> idsArray) {
    SearchRequest.Builder searchBuilder1 = new SearchRequest.Builder().index(config.getHotspotIndex()).query(q -> q.terms(t -> t.field("nm").
        terms(s -> s.value(idsArray.stream().map(y -> FieldValue.of(y)).collect(Collectors.toList()))))).size(idsArray.size());
    SearchRequest.Builder searchBuilder2 = new SearchRequest.Builder().index(config.getHotspotIndex()).query(q -> q.terms(t -> t.field("nm").
        terms(s -> s.value(idsArray.stream().map(y -> FieldValue.of(y)).collect(Collectors.toList()))))).size(idsArray.size());
    try {
      List<Hit<HotSpotNews>> value = getBatchResponse(searchBuilder1, searchBuilder2, -1).getValue();
      return value.stream().map(x->x.source()).collect(Collectors.toList());
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private Map<String, List<HotSpotNews>> wrapId2All(Map<String, List<String>> member2Id, List<HotSpotNews> newsFromIdsArray) {
    Map<String, HotSpotNews> newsfromIdMap = new LinkedHashMap<>();
    for (HotSpotNews hotSpotNews: newsFromIdsArray) {
      String id = hotSpotNews.getId();
      newsfromIdMap.put(id, hotSpotNews);
    }
    Map<String, List<HotSpotNews>> ret = new HashMap<>();
    for (String string: member2Id.keySet()) {
      ret.put(string, new ArrayList<>());
      List<String> strings = member2Id.get(string);
      for (String memberString: strings) {
        ret.get(string).add(newsfromIdMap.get(memberString));
      }
    }

    return ret;
  }

  private JSONObject wrapOneItem(HotSpotNews news,
                                 double sim
  ) {

      JSONObject tNews = new JSONObject();
      assert news != null;
      tNews.put("sjms", news.getNewsContentZh());
      tNews.put("zlnm", news.getMaterialId() == null ? "": news.getMaterialId());
      tNews.put("drsj", getDrsj(news));
      tNews.put("isNews", false);
      tNews.put("hwly", false);
      tNews.put("xh", 1);
      tNews.put("zmsj", false);
      tNews.put("same", false);
      tNews.put("fssj", news.getNewsPublicDate());
      tNews.put("sim", sim);
      tNews.put("xgss", "-");
      tNews.put("sjmc", news.getNewsTitleZh());
      tNews.put("ssly", getArea((String)news.getArea()));
      tNews.put("sird", false);
      tNews.put("psdj", "");
      tNews.put("lyss", news.getFromProvince());
      tNews.put("nm", news.getId());
      return tNews;
  }

  private String getArea(String area) {
    if (area == null) return "";
    if (config.getAreaCastMap().get(area) == null)   return area.substring(0, 4);
    return config.getAreaCastMap().get(area);
  }


  private JSONObject wrapRet(
                             Map<String, List<HotSpotNews>> all,
                             Map<String, List<Double>> castSim,
                             HotSpotParam queryParam) {
    JSONObject ret = new JSONObject();
    Map<String, JSONArray> cast =  new LinkedHashMap<>();
    for (String key: all.keySet()) {
        cast.put(key, new JSONArray());
        JSONArray jsonArray = cast.get(key);
        List<HotSpotNews> hotSpotNews = all.get(key);
        for (int i = 0; i < hotSpotNews.size(); i++) {
          jsonArray.add(wrapOneItem(hotSpotNews.get(i), castSim.get(key).get(i)));
        }
        cast.put(key, jsonArray);
    }
    List<JSONArray> tempArray
        = new ArrayList<>();
    // 簇内顺序
    for (String key: cast.keySet()) {
      JSONArray jsonArray = cast.get(key);
      // 簇内部先排序
//      sortClusterInner(jsonArray);

      tempArray.add(jsonArray);
    }
    // 簇间顺序排序
    List<List<JSONArray>> lists = split2Half(tempArray);
    sortClusterBetween(lists.get(0)); sortClusterBetween(lists.get(1));
    tempArray = new ArrayList<>();
    tempArray.addAll(lists.get(0)); tempArray.addAll(lists.get(1));

    // 包装data
    JSONArray retData = new JSONArray();
    for (int i= 0; i < tempArray.size(); i++) {
      JSONArray jsonArray = tempArray.get(i);
      JSONObject jsonObject = new JSONObject();
      jsonObject.put("cjsx", i+1);
      jsonObject.put("drsj", jsonArray.getJSONObject(0).getString("drsj"));
      jsonObject.put("hot", jsonArray.size());
      jsonObject.put("event", jsonArray);
      jsonObject.put("cuHasNew", false);
      retData.add(jsonObject);
    }
    int count = 0;
    for (JSONArray jsonArray: cast.values()) {
      count += jsonArray.size();
    }
    {
      ret.put("total", cast.size());
      ret.put("alive", true);
      ret.put("sim", queryParam.getSimWeight().toString());
      ret.put("entity", queryParam.getEntityWeight().toString());
      ret.put("count", count);
      ret.put("currentPage", queryParam.getCurrentPage());
      ret.put("pageSize", queryParam.getPageSize());
      ret.put("data", retData);
    }
    return ret;
  }

  private List<List<JSONArray>> split2Half(List<JSONArray> tempArray) {
    List<List<JSONArray>> ret = new ArrayList<>();
    Set<String> provinces = config.getProvince2Nm().keySet();
    List<JSONArray> provincesList = new ArrayList<>();
    List<JSONArray> departmentsList = new ArrayList<>();
    for (JSONArray jsonArray: tempArray) {
      String lyss = jsonArray.getJSONObject(0).getString("lyss");
      if (provinces.contains(lyss)) {
        provincesList.add(jsonArray);
      } else {
        departmentsList.add(jsonArray);
      }
    }
    ret.add(provincesList);
    ret.add(departmentsList);
    return ret;

  }

  private void sortClusterInner(JSONArray jsonArray) {
    jsonArray.sort((x, y)->{
      JSONObject jsonObjectX = (JSONObject) x ;
      JSONObject jsonObjectY = (JSONObject) y;
      String drsjX = jsonObjectX.getString("drsj");
      String drsjY = jsonObjectY.getString("drsj");
      Date timeX = RenderDateUtil.transTime(drsjX);
      Date timeY = RenderDateUtil.transTime(drsjY);
      return timeX.compareTo(timeY);
    });
  }


  private void sortClusterBetween( List<JSONArray> tempArray) {
    tempArray.sort((x, y)-> {
      int sizeX = x.size();
      int sizeY = y.size();
      if (sizeY == sizeX) {
        JSONObject jsonObjectX = x.getJSONObject(0);
        JSONObject jsonObjectY = y.getJSONObject(0);
        String drsjX = jsonObjectX.getString("drsj");
        String drsjY = jsonObjectY.getString("drsj");
        Date timeX = RenderDateUtil.transTime(drsjX);
        Date timeY = RenderDateUtil.transTime(drsjY);
        return timeX.compareTo(timeY);
      } else {
        return sizeY - sizeX;
      }
    });
  }

  private void sortClusterBetween2( List<List<JSONObject>>  nm2Array) {
    nm2Array.sort((x, y)-> {
      int sizeX = x.size();
      int sizeY = y.size();
      if (sizeY == sizeX) {
        JSONObject jsonObjectX = x.get(0);
        JSONObject jsonObjectY = y.get(0);
        String drsjX = jsonObjectX.getString("drsj");
        String drsjY = jsonObjectY.getString("drsj");
        Date timeX = RenderDateUtil.transTime(drsjX);
        Date timeY = RenderDateUtil.transTime(drsjY);
        return timeX.compareTo(timeY);
      } else {
        return sizeY - sizeX;
      }
    });
  }
  private void sortClusterBetween( JSONArray doc) {
    doc.sort((x,y)->{
      JSONObject jsonObjectX = (JSONObject) x;
      JSONObject jsonObjectY = (JSONObject) y;
      JSONArray eventX = jsonObjectX.getJSONArray("event");
      JSONArray eventY = jsonObjectY.getJSONArray("event");
      int sizeX = eventX.size();
      int sizeY = eventY.size();
      if (sizeY == sizeX) {
        JSONObject tempX = eventX.getJSONObject(0);
        JSONObject tempY = eventY.getJSONObject(0);
        String drsjX = tempX.getString("drsj");
        String drsjY = tempY.getString("drsj");
        Date timeX = RenderDateUtil.transTime(drsjX);
        Date timeY = RenderDateUtil.transTime(drsjY);
        return timeX.compareTo(timeY);
      }
      return sizeY - sizeX;
    });
  }



  private Map<String, List<HotSpotNews>> incrementCluster(Set<HotSpotNews> totalIncrement,
                                                          Map<String, List<HotSpotNews>> id2All,
                                                          double factor) {
    Set<String> departments = config.getDepartment2Nm().keySet();
    Set<String> provinces = config.getProvince2Nm().keySet();
    for (HotSpotNews hotSpotNews: totalIncrement) {
      boolean flag = true;
      List<Pair<String, Double>> pairs = new ArrayList<>();
      Data data1 = new Data("1", ResponseUtil.wrappedKeyword(getFilteredKeywords(hotSpotNews)), new HashSet<>(), "2022-01-01");
      String compareProvince = (String) hotSpotNews.getFromProvince();
      for (String id: id2All.keySet()) {
        List<HotSpotNews> hotSpotList = id2All.get(id);
        double sim = 0;
        for (HotSpotNews member: hotSpotList) {
          String memberProvince = (String) member.getFromProvince();
          if (!(provinces.contains(memberProvince) && provinces.contains(compareProvince)
//              || departments.contains(memberProvince) && departments.contains(compareProvince)
          )) {
            break;
          }
          Data data2 = new Data("2", ResponseUtil.wrappedKeyword(getFilteredKeywords(member)), new HashSet<>(), "2022-01-01");

          double tempSim = data1.calSim(data2);

          if (tempSim > sim)  sim = tempSim;
        }
        if (sim >= factor) {
          pairs.add(new Pair<>(id, sim));
          flag = false;
        }
      }
      if (flag) {
        List<HotSpotNews> temp = new ArrayList<>();
        temp.add(hotSpotNews);
        id2All.put(hotSpotNews.getId(), temp);
      } else {
        pairs.sort((x, y)-> {
          Double value1 = x.getValue();
          Double value2 = y.getValue();
          int ret;
          if (Objects.equals(value1, value2)) ret = 0;
          else {
            ret = value1 < value2 ? 1: -1;
          }
          return ret;
        });
        Pair<String, Double> pair = pairs.get(0);
        String id = pair.getKey();
        List<HotSpotNews> lists = id2All.get(id);
        lists.add(hotSpotNews);
      }
    }
    return id2All;
  }


  private void tempwrite(List<HotSpotNews> list) {
    String file_name = "temp_incre.txt";
    File file = new File(file_name);
    try {
      if (!file.exists()) {
        if (file.createNewFile()) {
          log.info("temp_incre success");
        } else {
          log.info("temp_incre error...");
        }
      }

      OutputStreamWriter osw = new OutputStreamWriter(
          Files.newOutputStream(Paths.get(file_name)),
          StandardCharsets.UTF_8);
      for (HotSpotNews hotSpotNews : list) {
        osw.write(hotSpotNews.toString());
      }
      osw.flush();
      osw.close();
    } catch (Exception e) {
      log.error("创建文件失败", e);
    }
  }

  // 获取聚类结果
  public Message getAssembleResult(HotSpotParam queryParam) throws Exception {
    // 1.参数检查
    TimeUtil totalTimeUtil = new TimeUtil();
    Map<String, Object> errMessage = HotSpotParamCheckUtil.checkParam(queryParam, config.getHotPageSize());
    String isSuccess = (String) errMessage.get("isSuccess");
    if (!isSuccess.equals("ok")) {
      log.error(isSuccess);
      return Message.unSuccess(isSuccess);
    }
    queryParam = HotSpotParam.getFromFormData(errMessage);
    Message message = null;
    JSONObject data = getResponse(queryParam);
//    log.info("28s ret: {}", data.toString());
    JSONObject dataJSONObject = data.getJSONObject("data");
    int total = (int)(dataJSONObject.get("total"));
//    JSONObject dataJSONObject = new JSONObject();
//    int total = 0;
    String startDate = queryParam.getStartDate();
    String endDate = queryParam.getEndDate();
    Set<String> distanceDates = new HashSet<>(RenderDateUtil.getDistanceDate(startDate, endDate));
    JSONObject ret = null;
    // 如果需要初始聚类
    if (total == 0) {
      ret = changeClusterInOrder(globalCluster(queryParam));
    } else {
      int nowTotal = getNowTotal(distanceDates, dataJSONObject);
      // 如果库内的当前日期数据，存在当前临时结果库中不存在的数据， 那么增量聚类
      // 反之，原封不动返回数据
      if (judgeIncrementCluster(queryParam, nowTotal)) {
        HotSpotParam finalQueryParam = queryParam;
        List<FieldValue> excludesProvince = buildRequest.getExclusiveProvince();
        SearchRequest.Builder builder1 = new SearchRequest.Builder().index(config.getHotspotIndex()).query(q -> q.bool(f -> f.filter(r -> r.range(t -> t.field("fssj")
            .gte(JsonData.of(startDate))
            .lte(JsonData.of(endDate))
        )).mustNot(k->k.terms(t->t.field("lyss").terms(m->m.value(excludesProvince)))))).size(finalQueryParam.getPageSize()).source(f -> f.filter(s -> s.includes("nm")));
        SearchRequest.Builder builder2 = new SearchRequest.Builder().index(config.getHotspotIndex()).query(q -> q.bool(f -> f.filter(r -> r.range(t -> t.field("fssj")
            .gte(JsonData.of(startDate))
            .lte(JsonData.of(endDate))
        ))
            .mustNot(k->k.terms(t->t.field("lyss").terms(m->m.value(excludesProvince))))
        )).size(finalQueryParam.getPageSize()).source(f -> f.filter(s -> s.includes("nm")));
        List<Hit<HotSpotNews>> hits = getBatchResponse(builder1, builder2, finalQueryParam.getPageSize()).getValue();

        Set<HotSpotNews> collectNews = hits.stream().map(x -> x.source()).collect(Collectors.toSet());
//        Map<String, HotSpotNews> collectNewsMap = new HashMap<>();
//        for (HotSpotNews hotSpotNews: collectNews) {
//          String id = hotSpotNews.getId();
//          collectNewsMap.put(id, hotSpotNews);
//        }
        Set<String> inDataBaseIds = collectNews.stream().map(m->m.getId()).collect(Collectors.toSet());
        Set<String> totalIncrementIds = Sets.difference(inDataBaseIds, getInTempIds(distanceDates, dataJSONObject)).stream().collect(Collectors.toSet());
        List<HotSpotNews> totalIncrement = getNewsFromIdsArray(new ArrayList<>(totalIncrementIds));
//        tempwrite(totalIncrement);
        if (!totalIncrement.isEmpty()) {
          Map<String, JSONArray> cast = getPastCast(dataJSONObject);
          Map<String, List<String>> allMember2Id = wrapId2All(cast);
          List<String> allId = new ArrayList<>();
          for (List<String> values: allMember2Id.values()) {
            allId.addAll(values);
          }
          List<HotSpotNews> newsFromIdsArray = getNewsFromIdsArray(allId);
          Map<String, List<HotSpotNews>> id2All = wrapId2All(allMember2Id, newsFromIdsArray);
          Map<String, List<HotSpotNews>> all = incrementCluster(new LinkedHashSet<>(totalIncrement), id2All, queryParam.getSimWeight());
          Map<String, List<Double>> castSim = new HashMap<>();
          for(String key: all.keySet()) {
            List<HotSpotNews> allMembers = all.get(key);
//            if (allMembers.size() == 1) {
//              List<Double> temp = new ArrayList<>();
//              temp.add(1.0);
//              castSim.put(key, temp);
//            } else {
              castSim.put(key, calSim(allMembers.get(0), allMembers.subList(1, allMembers.size())));
//            }
          }
          ret = wrapRet(all, castSim, queryParam);
        } else {
          ret = dataJSONObject;
        }
      } else {
        ret = dataJSONObject;
      }
    }
    log.info("total getAssembleResult cost: {}s", totalTimeUtil.getCost());
    return Message.builder().code(200).data(changeClusterInOrder(ret)).build();

  }

  private Map<String, List<String>> wrapId2All(Map<String, JSONArray> cast) {
    Map<String, List<String>> ret = new LinkedHashMap<>();
    for (String string: cast.keySet()) {
      JSONArray jsonArray = cast.get(string);
      ret.put(string, new ArrayList<>());
      for (Object o: jsonArray) {
        JSONObject jsonObject = (JSONObject) o;
        String nm = jsonObject.getString("nm");
        ret.get(string).add(nm);
      }
    }
    return ret;
  }

  private String wrapDate(String date) {
    if (Strings.isBlank(date)) {
      return "2010-01-01 00:00:00";
    } return date + " 00:00:00";
  }

  private JSONArray processDoc(Pair<Map<String, Hit<HotSpotNews>>, Map<String, Hit<HotSpotNews>>> map2News,
                               Pair<Map<String, List<Hit<HotSpotNews>>>, Map<String, List<Hit<HotSpotNews>>> > sameNewsMap) {
    Map<String, Hit<HotSpotNews>> map2NewsKey = map2News.getKey();
    Map<String, Hit<HotSpotNews>> map2NewsValue = map2News.getValue();
    Map<String, List<Hit<HotSpotNews>>> sameNewsMapKey = sameNewsMap.getKey();
    Map<String, List<Hit<HotSpotNews>>> sameNewsMapValue = sameNewsMap.getValue();
    int size = sameNewsMapKey.size();
    JSONArray jsonArray = processDoc(map2NewsKey, sameNewsMapKey, 1);
    jsonArray.addAll(processDoc(map2NewsValue, sameNewsMapValue, 1 + size));
    return jsonArray;
  }


  // 包装返回聚类doc
  private JSONArray processDoc(Map<String, Hit<HotSpotNews>> map2News,
                               Map<String, List<Hit<HotSpotNews>>> sameNewsMap, int count) {
    JSONArray doc = new JSONArray();

    ArrayList<Map.Entry<String, List<Hit<HotSpotNews>>>> entries = new ArrayList<>(sameNewsMap.entrySet());
    for (Map.Entry<String, List<Hit<HotSpotNews>>> entry: entries) {
      String key = entry.getKey();
      HotSpotNews core = map2News.get(key).source();
      JSONObject jsonObject = new JSONObject();
      // 包装共有的
//      jsonObject.put("cjsx", count++);
      assert core != null;
      jsonObject.put("drsj", getDrsj(core));
      jsonObject.put("hot", sameNewsMap.get(key).size() + 1);
      // 包装event 数组
      List<Hit<HotSpotNews>> temp = new ArrayList<>();
      temp.add(map2News.get(key));
      temp.addAll(sameNewsMap.get(key));

      JSONArray event = new JSONArray();
      int count2 = 0;
      // 计算与核心点之间的距离
      List<Double> all = calSim(map2News.get(key).source(), sameNewsMap.get(key).stream().map(x->x.source()).collect(Collectors.toList()));

      for (int i = 0 ;i < temp.size(); i++) {
        Hit<HotSpotNews> h = temp.get(i);
        HotSpotNews news = h.source();
        JSONObject tNews = new JSONObject();
          assert news != null;
          tNews.put("sjms", news.getNewsContentZh());
        tNews.put("zlnm", news.getMaterialId() == null ? "": news.getMaterialId());

        tNews.put("drsj", i == 0 ? jsonObject.getString("drsj") : getDrsj(news));
        tNews.put("isNews", false);
        tNews.put("hwly", false);
        tNews.put("xh", 1);
        tNews.put("zmsj", false);
        tNews.put("same", false);
        tNews.put("fssj", news.getNewsPublicDate());
        tNews.put("sim", all.get(count2++));
        tNews.put("xgss", "-");
        tNews.put("sjmc", news.getNewsTitleZh());
        tNews.put("ssly", getArea((String) news.getArea()));
        tNews.put("sird", false);
        tNews.put("psdj", "");
        tNews.put("lyss", news.getFromProvince());
        tNews.put("nm", news.getId());
        event.add(tNews);
      }
      sortClusterInner(event);
      jsonObject.put("event", event);
      doc.add(jsonObject);
    }
    sortClusterBetween(doc);
    for (int i = 0 ; i < doc.size(); i++) {
      JSONObject jsonObject = (JSONObject) doc.get(i);
      jsonObject.put("cjsx", i +  count);
    }

    return doc;
  }

  private String getDrsj(HotSpotNews news) {
    return Strings.isBlank(news.getDrsj()) ? RenderDateUtil.getNow(): news.getDrsj();
  }

  private List<Double> calSim(
                              HotSpotNews core,
                              List<HotSpotNews> member) {
    List<Double> ret = new ArrayList<>();
    ret.add(1.0);
    Data coreData = wrapNews2Data(core);
    for (HotSpotNews news: member) {

      Data data = wrapNews2Data(news);
      ret.add(coreData.calSim(data));
    }

    return ret;
  }

  private Data wrapNews2Data(HotSpotNews core) {
    return new Data(core.getId(), ResponseUtil.wrappedKeyword(getFilteredKeywords(core)),
        ResponseUtil.wrappedKeyword(core.getNewsEntities()), core.getNewsPublicDate());
  }


  private List<Hit<HotSpotNews>> distinctById(List<Hit<HotSpotNews>> list) {
    Map<String, Hit<HotSpotNews>> set = new LinkedHashMap<>();
    for (Hit<HotSpotNews> hit: list) {
      assert hit.source() != null;
      set.put(hit.source().getId(), hit);
    }
    return new ArrayList<>(set.values());
  }

  private Map<String, HotSpotNews> getSeveralNewsById(List<String> ids) {
    Map<String, HotSpotNews> ret = new LinkedHashMap<>();
    SearchRequest searchRequest = SearchRequest.of(x -> x.index(config.getHotspotIndex()).size(maxSize).query(
        q -> q.terms(t -> t.field("nm").terms(tt -> tt.value(ids.stream().map(temp -> FieldValue.of(temp)).collect(Collectors.toList()))))
    ));


    try {
      SearchResponse<HotSpotNews> r = elasticsearchClient.search(searchRequest, HotSpotNews.class);
      log.info("request news: {}", searchRequest);
      List<HotSpotNews> collect = r.hits().hits().stream().map(x -> x.source()).collect(Collectors.toList());
      for (HotSpotNews hotSpotNews: collect) {
        String id = hotSpotNews.getId();
        ret.put(id, hotSpotNews);
      }
      return ret;
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  public Message addHotsToCrowd(HotMissionParam queryParam) {
    TimeUtil totalTimeUtil = new TimeUtil();
    Map<String, Object> errMessage = HotMissionParamCheck.checkParam(queryParam);
    String isSuccess = (String) errMessage.get("isSuccess");
    if (!isSuccess.equals("ok")) {
      log.error(isSuccess);
      return Message.unSuccess(isSuccess);
    }
    queryParam = (HotMissionParam) errMessage.get("formData");
    HotSpotParam hotSpotParam = new HotSpotParam();
    hotSpotParam.setNm(queryParam.getRwnm());
    JSONObject data = getResponse(hotSpotParam);
    String nm = queryParam.getNm();
    // 过滤实际需要添加的（要添加的数据可能和暂存库里的重合）
    List<String> nmsArray = queryParam.getNmsArray();
    JSONObject dataJSONObject = data.getJSONObject("data");
    Map<String, JSONArray> cast = getPastCast(dataJSONObject);
    Map<String, Map<String, JSONObject>> nm2JsonMap =  wrapNm2JsonMap(cast);
    // 找到真正的core
    nm = findCore(nm, nm2JsonMap);
    Map<String, String> nm2Center = id2Center(nm2JsonMap, nmsArray);
    Map<String, HotSpotNews> severalNewsById = getSeveralNewsById(nmsArray);
    // 如果有指向簇(nm不为空)
    String finalNm = nm;


    if (Strings.isNotBlank(nm)) {
      HotSpotNews coreNews = getSeveralNewsById(new ArrayList<String>() {{
        add(finalNm);
      }}).get(nm);
      if (coreNews == null) {
        return Message.builder().code(200).data(data).build();
      }
      log.info("severalNewsById: {}", severalNewsById);
      List<Double> calledSim = calSim(coreNews, new ArrayList<>(severalNewsById.values()));
      log.info("calledSim： {}", calledSim);

      calledSim.remove(0);
      int tempI = 0;

      for (String string: severalNewsById.keySet()) {
        if (nm2JsonMap.get(nm).containsKey(string)) continue;
        if (nm2Center.containsKey(string)) {
          String father = nm2Center.get(string);
          nm2JsonMap.get(father).remove(string);
        }
        nm2JsonMap.get(nm).put(string, wrapOneItem(severalNewsById.get(string), calledSim.get(tempI++)));
      }
      // 判断nm2JsonMap中
      Set<String> set = new HashSet<>();
      for (String key: nm2JsonMap.keySet()) {
        if (nm2JsonMap.get(key).isEmpty()) {
          set.add(key);
        }
      }
      for (String string: set) {
        nm2JsonMap.remove(string);
      }
      JSONObject ret = new JSONObject();
      ret.put("total", nm2JsonMap.size());
      JSONArray retArray = new JSONArray();

      List<List<JSONObject>> nm2Array = new ArrayList<>();
      for (Map<String, JSONObject> map: nm2JsonMap.values()) {
        ArrayList<JSONObject> temp = new ArrayList<>(map.values());
//        temp.sort((x, y)-> {
//          String drsjX = x.getString("drsj");
//          String drsjY = y.getString("drsj");
//          Date timeX = RenderDateUtil.transTime(drsjX);
//          Date timeY = RenderDateUtil.transTime(drsjY);
//          return timeX.compareTo(timeY);
//        });
        nm2Array.add(temp);
      }
      sortClusterBetween2(nm2Array);
      int countAll = 0;

      for (int i = 0; i < nm2Array.size(); i++) {
        List<JSONObject> jsonObjects = nm2Array.get(i);
        JSONObject temp = new JSONObject();
        JSONArray tempJsonArray = new JSONArray();
        for (int j = 0; j < jsonObjects.size(); j++) {
          JSONObject tt = jsonObjects.get(j);
          tempJsonArray.add(tt);
          countAll ++;
        }
        temp.put("drsj", jsonObjects.get(0).getString("drsj"));
        temp.put("cjsx", i + 1);
        temp.put("hot", jsonObjects.size());
        temp.put("cuHasNew", false);
        temp.put("event", tempJsonArray);
        retArray.add(temp);
      }

      ret.put("data", retArray);
      ret.put("total", nm2Array.size());
      ret.put("alive", true);
      ret.put("sim", dataJSONObject.get("sim"));
      ret.put("entity", dataJSONObject.get("entity"));
      ret.put("count", countAll);
      ret.put("currentPage", dataJSONObject.getInteger("currentPage"));
      ret.put("pageSize", dataJSONObject.getInteger("pageSize"));
      log.info("total addHotsToCrowd cost: {}s", totalTimeUtil.getCost());
      return Message.builder().code(200).data(changeClusterInOrder(ret)).build();
    } else {
      Set<String> actualSet = new LinkedHashSet<>(nmsArray);
      for (String string : severalNewsById.keySet()) {
        if (nm2Center.containsKey(string)) {
//          String father = nm2Center.get(string);
//          nm2JsonMap.get(father).remove(string);
//          nm2Center.remove(string);
          actualSet.remove(string);
        }
      }
      // 判断nm2JsonMap中
      Set<String> set = new HashSet<>();
      for (String key : nm2JsonMap.keySet()) {
        if (nm2JsonMap.get(key).isEmpty()) {
          set.add(key);
        }
      }
      for (String string : set) {
        nm2JsonMap.remove(string);
      }
      cast = new LinkedHashMap<>();
      for (String key : nm2JsonMap.keySet()) {
        cast.put(key, new JSONArray(new ArrayList<>(nm2JsonMap.get(key).values())));
      }
      List<HotSpotNews> actualNews = getNewsFromIdsArray(new ArrayList<>(actualSet));
      Map<String, List<String>> allMember2Id = wrapId2All(cast);
      List<String> allId = new ArrayList<>();
      for (List<String> values: allMember2Id.values()) {
        allId.addAll(values);
      }
      List<HotSpotNews> newsFromIdsArray = getNewsFromIdsArray(allId);
      Map<String, List<HotSpotNews>> id2All = wrapId2All(allMember2Id, newsFromIdsArray);
      Map<String, List<HotSpotNews>> all = incrementCluster(new LinkedHashSet<>(actualNews), id2All, queryParam.getSimWeight());

      Map<String, List<Double>> castSim = new HashMap<>();

      for (String key : all.keySet()) {
        List<HotSpotNews> allMembers = all.get(key);
        castSim.put(key, calSim(allMembers.get(0), allMembers.subList(1, allMembers.size())));
      }
      HotSpotParam param = new HotSpotParam();
      param.setSimWeight(Double.parseDouble(dataJSONObject.getString("sim")));
      param.setEntityWeight(Double.parseDouble(dataJSONObject.getString("entity")));
      param.setCurrentPage(dataJSONObject.getInteger("currentPage"));
      param.setPageSize(dataJSONObject.getInteger("pageSize"));
      JSONObject ret = wrapRet(all, castSim, param);
      log.info("total addHotsToCrowd cost: {}s", totalTimeUtil.getCost());
      return Message.builder().code(200).data(changeClusterInOrder(ret)).build();
    }
  }

  private String findCore(String nm, Map<String, Map<String, JSONObject>> nm2JsonMap) {
    String ret = nm;
    for (String string: nm2JsonMap.keySet()) {
      if (nm2JsonMap.get(string).containsKey(nm)) {
        ret = string;
        break;
      }
    }
    return ret;
  }

  private Map<String, String> id2Center(Map<String, Map<String, JSONObject>> nm2JsonMap,
                                        List<String> nmsArray) {
    Map<String, String> ret = new LinkedHashMap<>();
    Set<String> nmSet = new HashSet<>(nmsArray);
    for (String string: nm2JsonMap.keySet()) {
      Map<String, JSONObject> stringJSONObjectMap = nm2JsonMap.get(string);
      for (String key: stringJSONObjectMap.keySet()) {
        if (nmsArray.contains(key)) {
          ret.put(key, string);
        }
      }
    }
    return ret;
  }

  private Map<String, JSONObject> wrapNm2JsonArray(JSONArray jsonArray) {
    Map<String, JSONObject> ret = new LinkedHashMap<>();
    for (Object o: jsonArray) {
      JSONObject jsonObject = (JSONObject) o;
      ret.put(jsonObject.getString("nm"), jsonObject);
    }
    return ret;
  }

  private Map<String, Map<String, JSONObject>> wrapNm2JsonMap(Map<String, JSONArray> cast) {
    Map<String, Map<String, JSONObject>> ret = new LinkedHashMap<>();
    for (String string: cast.keySet()) {
      JSONArray jsonArray = cast.get(string);
      ret.put(string, wrapNm2JsonArray(jsonArray));
    }
    return ret;
  }


  public Message getStatisticsView(HotMissionParam param) {
    TimeUtil totalTimeUtil = new TimeUtil();
    String rwnm = param.getRwnm();
    HotSpotParam hotSpotParam = new HotSpotParam();
    hotSpotParam.setNm(rwnm);
    JSONObject data = getResponse(hotSpotParam);
    JSONObject dataJSONObject = data.getJSONObject("data");
    Map<String, JSONArray> cast = getPastCast(dataJSONObject);
    Map<String, String> province2Nm = config.getProvince2Nm();
    Map<String, String> department2Nm = config.getDepartment2Nm();
    Map<String, Integer> provinceCount = new LinkedHashMap<>();
    Map<String, Integer> departmentCount = new LinkedHashMap<>();
    for (JSONArray array: cast.values()) {
      for (Object o: array) {
        JSONObject jsonObject = (JSONObject) o;
        String lyss = jsonObject.getString("lyss");
        if (province2Nm.containsKey(lyss)) {
          provinceCount.compute(lyss, (key, value) -> (value == null) ? 1 : value + 1 );
        }
        if (department2Nm.containsKey(lyss)) {
          departmentCount.compute(lyss, (key, value) -> (value == null) ? 1 : value + 1 );
        }
      }
    }
    JSONObject ret = new JSONObject();
    ret.put("source1", wrapSource(department2Nm, departmentCount, "部门热点"));
    ret.put("source", wrapSource(province2Nm, provinceCount, "地方热点"));
    return Message.builder().code(200).data(ret).message("success").build();


  }

  /**
   * 包装返回给热点目录的不同source
   * @param nmMap
   * @param countMap
   * @return
   */
  private List<JSONObject> wrapSource(Map<String, String> nmMap,
                                      Map<String, Integer> countMap,
                                      String type) {
    // 部门总数
    Integer total;
    if (countMap.values().isEmpty()) {
      total = 0;
    } else {
      total = countMap.values().stream().reduce((a, b) -> a + b).get();
    }

    // 部门热点
    List<JSONObject> source = new ArrayList<>();
    source.add(new JSONObject(){{
      put("name", type);
      put("count", total + "/" + total);
      put("sl", total);
      put("nm", "00");
    }});
    for (String string: nmMap.keySet()) {
      int count = 0;
      if (countMap.containsKey(string)) {
        count = countMap.get(string);
      }
      int finalCount = count;
      source.add(new JSONObject(){{
        put("name", string);
        if (finalCount != 0) {
          put("count", finalCount + "/" + finalCount);
        } else {
          put("count", "");
        }
        put("sl", finalCount);
        put("nm", nmMap.get(string));
      }});
    }
    source.sort((x,y)->{
      int ix = x.getInteger("sl");
      int iy = y.getInteger("sl");
      return iy - ix;
    });
    return source;
  }


  /**
   * 更改簇内顺序
   */
  private JSONObject changeClusterInOrder(JSONObject jsonObject) {
    List<Object> data = jsonObject.getJSONArray("data");
    for (Object o: data) {
      JSONObject d = (JSONObject) o;
      List<JSONObject> e = (List<JSONObject>)d.get("event");
      int count = 1;
      for (JSONObject t: e) {
        t.put("cnsx", count++);
      }
    }
    return jsonObject;
  }
}
